package com.eliansun.crawler.pageparse.chongqing;

import com.eliansun.crawler.pageparse.AbstractPageProcess;
import net.dongliu.requests.Requests;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;

import java.util.HashMap;
import java.util.Map;

/**
 * user: lxc
 * data:2019- 07-04-09
 * 云采医药平台     www.yuncai998.com
 */
public class Yuncai extends AbstractPageProcess {

    int count = 0;

    public static String POST_URL = "https://www.yuncai998.com/product/search";

    @Override
    public void process(Page page) {
        try{
            //列表界面
            if( page.getUrl().toString().contains("list") ){
                Map<String,Object> parmes = new HashMap<String,Object>(16);
                parmes.put("pageNo","1");
                String result = Requests.post(POST_URL).params(parmes).send().readToText();
                Document parse = Jsoup.parse(result);
                Elements elements =parse.getElementsByClass("body").get(1).getElementsByTag("a");
                for (int i = 0; i < elements.size(); i++) {
                    String href = elements.attr("href");
                    page.addTargetRequest( href );
                }
                



            }else{
                //详情界面
                Map<String,Object> resMap = new HashMap<String,Object>(16);
               // 商品名称
               resMap.put("goodsName",  page.getHtml().xpath("//div[@class=title]/h1/text()"));




              page.putField("result", resMap);

              count ++;
              System.out.println( resMap + "----第" +  count );
           }
        }catch ( Exception e){
               e.printStackTrace();
        }
    }

    Site site = Site.me()
            //设置编码
            .setCharset("utf-8")
            //设置抓取间隔时间
            .setSleepTime(1000*3)
            //设置超时时间，单位毫秒
            .setTimeOut(1000*3)
            //设置重试间隔时间
            .setRetrySleepTime(1000*3)
            //设置重试次数
            .setRetryTimes(3);
    @Override
    public Site getSite() {
        return site;
    }



    public static void main(String[] args) {
        Spider.create( new Yuncai())
               // .addUrl("http://www.eycx.com/product/detail/65283/")
                .addUrl("https://www.yuncai998.com/product/list")
                //设置多个线程同时抓取数据
                .thread(1)
                .run();

    }

}
